package com.universityJob.modules.job.task;

import com.google.common.collect.Lists;
import com.google.gson.Gson;
import com.universityJob.common.utils.UUIDUtil;
import com.universityJob.modules.data.entity.DataJobEntity;
import com.universityJob.modules.data.entity.DataRecEntity;
import com.universityJob.modules.data.service.DataJobService;
import com.universityJob.modules.data.service.DataRecService;
import org.apache.commons.collections.MapUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * @ClassName: Get12306
 * @Description
 * @Author maozhi
 * @Date 2021/4/23
 * @Time 18:16
 */
@Component("getChangAnUniversity")
public class GetChangAnUniversity implements ITask {

    @Autowired
    DataRecService dataRecService;
    @Autowired
    DataJobService dataJobService;

    @Autowired
    UUIDUtil uuidUtil;

    private Logger logger = LoggerFactory.getLogger(getClass());
    private static Map<String, String> headers = new HashMap<>();
    private int page;
    private String recTags;
    private String companySize;


    public void setRecTags(String recTags) {
        this.recTags = recTags;
    }

    public void setCompanySize(String companySize) {
        this.companySize = companySize;
    }
    public void setPage(int page) {
        this.page = page;
    }


    {
        {
            headers.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
            headers.put("Accept-Encoding", "gzip, deflate, br");
            headers.put("Content-Type", "application/json;charset=UTF-8");
            headers.put("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
            headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36");
        }
    }

    private static String[] USER_AGENTS = {
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36",
            "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
            "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
            "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
            "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
            "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
    };

    public static int getCount() {
        String urls = "http://xsjyc.chd.edu.cn/Frame/Data/jdp.ashx?rnd=1619162409088&fn=GetZhaopinList" +
                "&StartDate=2122-01-01&SearchKey=&InfoType=-1&CompanyAttr=" +
                "&CompanyType=&Area=&City=&CompanyProvice=&Post=&Zhuanye=&XLkey=&Age=&start=1" +
                "&limit=1&DateType=999&InfoState=1&WorkType=0&CompanyKey=";
        Document bo = getDoc(urls);
        String text = bo.text();
        Map<String, Object> stringObjectMap = jsonToMap(text);
        Double total = (Double) stringObjectMap.get("total");
        int count = total.intValue();
        return count;
    }

    /**
     * 页面爬取
     *
     * @param url
     * @return
     */
    public static Document getDoc(String url) {
        Document doc = null;
        try {
            Random ra = new Random();
            int i = ra.nextInt(10);
            headers.put("User-Agent", USER_AGENTS[i]);
            doc = Jsoup.connect(url).headers(headers).ignoreContentType(true).timeout(6000).get();
            return doc;
        } catch (IOException e) {
            e.printStackTrace();
        }
        return doc;
    }

    /**
     * job页面内容爬取
     *
     * @param jobOid
     * @return
     */
    public Map getContent(String jobOid) {
        Map map = new HashMap();
        String url = "https://xsjyc.chd.edu.cn/Frame/Data/jdp.ashx?rnd=1619269243670&fn=GetOneZhaopin&JobId="+ jobOid+"&StartDate=2000-01-01" ;
        Document doc = getDoc(url);
        Map<String, Object> stringObjectMap = jsonToMap(doc.text());
        Map data = (Map) stringObjectMap.get("Data");
        map.put("jobContent", MapUtils.getString(data,"EmployContent",""));
        map.put("CompanySizeName", MapUtils.getString(data,"CompanySizeName",""));
        map.put("recTags",MapUtils.getString(data,"WorkPrice",""));
        return map;
    }

    /**
     * 获取rec实体类数据
     */
    public DataRecEntity getRec(Map map) {
        DataRecEntity dataRecEntity = new DataRecEntity();
//        UUIDUtil uuidUtil = new UUIDUtil();
        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
        String publishDate = MapUtils.getString(map, "AddTime");
        String updateDate = MapUtils.getString(map, "AddTime");
        try {
            dataRecEntity.setPublishdate(simpleDateFormat.parse(publishDate));
            dataRecEntity.setUpdatedate(simpleDateFormat.parse(updateDate));
        } catch (ParseException e) {
        }
        dataRecEntity.setOldid(MapUtils.getString(map, "CompanyId"));
        dataRecEntity.setRecid(uuidUtil.getCharAndNumr(16));
        dataRecEntity.setReclogo("");
        dataRecEntity.setRecname(MapUtils.getString(map, "CompanyTitle", ""));
        dataRecEntity.setRecproperty(MapUtils.getString(map, "TypeName", ""));

        return dataRecEntity;
    }

    /**
     * 获取job实体类数据
     */
    public DataJobEntity getJob(Map map, DataRecEntity rec) {
        DataJobEntity dataJobEntity = new DataJobEntity();
        String areaCodeName = MapUtils.getString(map, "CompanyAreaName");
        StringBuilder sb = new StringBuilder(areaCodeName);
//        UUIDUtil uuidUtil = new UUIDUtil();
        String insert = null;
        if (areaCodeName.contains("省")) {
            int i = areaCodeName.indexOf("省");
            insert = sb.insert(i + 1, ",").toString();
        } else if (areaCodeName.contains("市")) {
            int i = areaCodeName.indexOf("市");
            insert = sb.insert(i + 1, ",").toString();
        } else {
            insert = areaCodeName;
        }
        Map map1 = getContent(MapUtils.getString(map, "Id"));
        //设置rec
        rec.setRectags(MapUtils.getString(map1,"recTags",""));
        rec.setRecscale(MapUtils.getString(map1,"CompanySizeName","未知"));

        String publishDate = MapUtils.getString(map, "AddTime");
        String updateDate = MapUtils.getString(map, "AddTime");
        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
        try {
            dataJobEntity.setPublishdate(simpleDateFormat.parse(publishDate));
            dataJobEntity.setUpdatedate(simpleDateFormat.parse(updateDate));
        } catch (ParseException e) {
        }

        dataJobEntity.setOldid(MapUtils.getString(map, "Id"));
        dataJobEntity.setRecid(rec.getRecid());
        String jobid = uuidUtil.getCharAndNumr(16);
        dataJobEntity.setJobid(jobid);
        dataJobEntity.setJobname(MapUtils.getString(map, "WorkPositon", ""));
        dataJobEntity.setJobcontent(MapUtils.getString(map1, "jobContent"));
        dataJobEntity.setAreacodename(insert);
        dataJobEntity.setDegreename(MapUtils.getString(map, "WorkEdu", "无"));
        dataJobEntity.setHeadcount(MapUtils.getString(map, "EmployNum", "若干"));
        dataJobEntity.setHighmonthpay(MapUtils.getString(map, "WorkPrice", "面谈"));
        dataJobEntity.setLowmonthpay(MapUtils.getString(map, "lowMonthPay", "0"));
        dataJobEntity.setMajor(MapUtils.getString(map, "major", "无"));
        return dataJobEntity;
    }

    /**
     * string json转 Map对象
     *
     * @param jsonString
     * @return
     */
    public static Map<String, Object> jsonToMap(String jsonString) {
        Gson gson = new Gson();
        Map map = new HashMap();
        map = gson.fromJson(jsonString, map.getClass());
        return map;
    }

    public Map<String, List<Map<String, Object>>> jsonToMapList(String jsonString) {
        Gson gson = new Gson();
        Map map = new HashMap();
        map = gson.fromJson(jsonString, map.getClass());
        return map;
    }

    /**
     * 定时任务入口
     *
     * @param params 参数，多参数使用JSON数据
     */
    @Override
    public void run(String params) {
        logger.info("爬取任务正在执行，参数为：{}", params);
        int count = getCount();
        logger.info("爬取到数据，共：{} 页", count);
        logger.info("------------开始清除rec表数据--------------");
//        dataRecService.truncateRec();
        logger.info("------------清除rec表数据成功--------------");
        logger.info("------------开始清除job表数据--------------");
//        dataJobService.truncateJob();
        logger.info("------------清除job表数据成功--------------");
        logger.info("-----------开始爬取页面内容数据----------");
        //执行线程
        int TOTAL_THREADS = 15;//线程数
        if (count <= TOTAL_THREADS){
            TOTAL_THREADS = count;
        }
        //Executors创建线程池new固定的10个线程
        ExecutorService taskExecutor = Executors.newFixedThreadPool(TOTAL_THREADS);
        //数据先保存到内存中，之后统一进行插入
        List<DataJobEntity> jobs = new ArrayList<>(1000);
        List<DataRecEntity> recs = new ArrayList<>(1000);
        for (int m = 0; m < (count / 500 + 1); m++) {
            this.page = m * 500;
            final int n = this.page;

            Runnable run = new Runnable() {
                @Override
                public void run() {
                    String urlCount = "http://xsjyc.chd.edu.cn/Frame/Data/jdp.ashx?rnd=1619162409088&fn=GetZhaopinList" +
                            "&StartDate=2122-01-01&SearchKey=&InfoType=-1&CompanyAttr=" +
                            "&CompanyType=&Area=&City=&CompanyProvice=&Post=&Zhuanye=&XLkey=&Age=&start=" + n +
                            "&limit=500&DateType=999&InfoState=1&WorkType=0&CompanyKey=";
                    logger.info("正在爬取从: {} 后500条数据", n);


                    Document doc = getDoc(urlCount);

                    String text = doc.text();

                    Map<String, List<Map<String, Object>>> jsonToMapList = jsonToMapList(text);
                    List<Map<String, Object>> mapList = jsonToMapList.get("rows");

                    for (Map<String, Object> map : mapList) {
                        String recId = MapUtils.getString(map, "CompanyId");
                        String jobId = MapUtils.getString(map, "Id");
                        DataRecEntity recEntity = null;
                        try {
                            recEntity = dataRecService.selectRecByOid(recId);
                        } catch (Exception e) {
//                            e.printStackTrace();
                        }
                        if (recEntity == null) {
                            DataJobEntity dataJobEntity = null;
                            try {
                                dataJobEntity = dataJobService.selectJobByOid(jobId);
                            } catch (Exception e) {
//                                e.printStackTrace();
                            }
                            DataRecEntity rec = getRec(map);
                            if (dataJobEntity == null) {

                                DataJobEntity job = getJob(map, rec);

                                jobs.add(job);
                                recs.add(rec);
//                                dataJobService.saveJob(job);
//                                dataRecService.saveRec(rec);
                            } else {
                                recs.add(rec);
//                                dataRecService.saveRec(rec);
                                continue;
                            }
                        } else {
                            if (dataJobService.selectJobByOid(jobId) == null) {
                                DataJobEntity job = getJob(map, recEntity);
                                jobs.add(job);
//                                dataJobService.saveJob(job);
                            } else {
                                continue;
                            }
                        }
                    }
                }
            };
            try {
                Thread.sleep(2000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            taskExecutor.execute(run);//开启线程执行池中的任务。还有一个方法submit也可以做到，它的功能是提交指定的任务去执行并且返回Future对象，即执行的结果

        }
        taskExecutor.shutdown();//关闭线程池

        while (true) {
            if (taskExecutor.isTerminated()) {
                break;
            }
        }


        //所有线程执行完毕,执行主线程
        logger.info("-----------页面数据爬取完成数据----------");
        logger.info("-----------开始数据保存----------------");
        logger.info("-----------JOB{}----------------", jobs.size());
        logger.info("-----------REC{}----------------", recs.size());
        for (DataJobEntity job : jobs) {
            try {
                dataJobService.save(job);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        for (DataRecEntity rec : recs) {
            try {
                dataRecService.save(rec);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    public static void main(String[] args) {
        GetChangAnUniversity getChangAnUniversity = new GetChangAnUniversity();
        getChangAnUniversity.getContent("ae0c3ae497ee400db68f1b55562ec3c7");
    }
}
